#ifndef __PAGELIBPREPROCESSOR_HPP__
#define __PAGELIBPREPROCESSOR_HPP__

#include <vector>
#include <string>
#include <unordered_map>

using std::vector;
using std::string;
using std::pair;
using std::unordered_map;

class WordSegmentation;
class WebPage;
class Configuration;

//网页库预处理类
class PageLibPreprocessor
{
public:
    PageLibPreprocessor(Configuration& conf);//构造函数
    ~PageLibPreprocessor() {}//析构函数

    void doProcess();//执行预处理

private:
    void readInfoFromFile();//根据配置信息读取网页库和网页偏移库
    void cutRedundantPages();//对冗余的网页去重
    void buildInvertIndexTable();//创建倒排索引表
    void storeOnDisk();//将经过预处理后的网页库、网页偏移库和倒排索引表写会磁盘

private:
    WordSegmentation& _jieba;//分词对象的引用
    vector<WebPage> _pageLib;//网页库的容器对象
    unordered_map<int,pair<int,int>> _offsetLib;//网页偏移库对象
    unordered_map<string,vector<pair<int,double>>> _invertIndexTable;//倒排索引表对象

};


#endif
